{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "dd74faf3-13fb-44fa-875d-08e4b2bbbdbd",
   "metadata": {},
   "source": [
    "# Fasttext and Logistic Regression, 20 April 2023\n",
    "This notebook applies two of the models that were applied in the previous iteration of TRAM."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "327acf59-d053-4f37-b0db-386a59f67e5a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>technique_name</th>\n",
       "      <th>attack_id</th>\n",
       "      <th>subclass_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>has overwritten the function pointer in the ex...</td>\n",
       "      <td>Extra Window Memory Injection</td>\n",
       "      <td>T1055</td>\n",
       "      <td>011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>overwrites Explorers Shell_TrayWnd extra windo...</td>\n",
       "      <td>Extra Window Memory Injection</td>\n",
       "      <td>T1055</td>\n",
       "      <td>011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>has used scheduled tasks to maintain persistence.</td>\n",
       "      <td>Scheduled Task</td>\n",
       "      <td>T1053</td>\n",
       "      <td>005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>has the ability to launch scheduled tasks to e...</td>\n",
       "      <td>Scheduled Task</td>\n",
       "      <td>T1053</td>\n",
       "      <td>005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>has used scheduled tasks to maintain persistence.</td>\n",
       "      <td>Scheduled Task</td>\n",
       "      <td>T1053</td>\n",
       "      <td>005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24599</th>\n",
       "      <td>\"My God\" was one of the first songs recorded b...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24600</th>\n",
       "      <td>It initially had seven students.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24601</th>\n",
       "      <td>Vellarikundu is a hillside town and taluk head...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24602</th>\n",
       "      <td>This earned the score a parental advisory warn...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24603</th>\n",
       "      <td>Acanthocardamum is a monotypic genus in the fa...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24604 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    text   \n",
       "0      has overwritten the function pointer in the ex...  \\\n",
       "1      overwrites Explorers Shell_TrayWnd extra windo...   \n",
       "2      has used scheduled tasks to maintain persistence.   \n",
       "3      has the ability to launch scheduled tasks to e...   \n",
       "4      has used scheduled tasks to maintain persistence.   \n",
       "...                                                  ...   \n",
       "24599  \"My God\" was one of the first songs recorded b...   \n",
       "24600                   It initially had seven students.   \n",
       "24601  Vellarikundu is a hillside town and taluk head...   \n",
       "24602  This earned the score a parental advisory warn...   \n",
       "24603  Acanthocardamum is a monotypic genus in the fa...   \n",
       "\n",
       "                      technique_name attack_id subclass_id  \n",
       "0      Extra Window Memory Injection     T1055         011  \n",
       "1      Extra Window Memory Injection     T1055         011  \n",
       "2                     Scheduled Task     T1053         005  \n",
       "3                     Scheduled Task     T1053         005  \n",
       "4                     Scheduled Task     T1053         005  \n",
       "...                              ...       ...         ...  \n",
       "24599                            NaN       NaN         NaN  \n",
       "24600                            NaN       NaN         NaN  \n",
       "24601                            NaN       NaN         NaN  \n",
       "24602                            NaN       NaN         NaN  \n",
       "24603                            NaN       NaN         NaN  \n",
       "\n",
       "[24604 rows x 4 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "\n",
    "data_path = '/projects/TRAM2023/tram-private/data/training/refreshed_dataset_march_2023.json'\n",
    "with open(data_path) as f:\n",
    "    data = json.loads(f.read())\n",
    "\n",
    "raw = pd.DataFrame(data['sentences'])\n",
    "mappings = raw['mappings'].explode().dropna().apply(pd.Series)\n",
    "df = pd.concat((raw['text'], mappings['technique_name'], mappings['attack_id'].str.extract(r\"(?P<attack_id>T\\d+)\\.(?P<subclass_id>\\d+)\")), axis=1)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ece10f20-a038-4375-bda4-4152f39a3478",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>technique_name</th>\n",
       "      <th>attack_id</th>\n",
       "      <th>subclass_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>has overwritten the function pointer in the ex...</td>\n",
       "      <td>Extra Window Memory Injection</td>\n",
       "      <td>T1055</td>\n",
       "      <td>011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>overwrites Explorers Shell_TrayWnd extra windo...</td>\n",
       "      <td>Extra Window Memory Injection</td>\n",
       "      <td>T1055</td>\n",
       "      <td>011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>has used scheduled tasks to maintain persistence.</td>\n",
       "      <td>Scheduled Task</td>\n",
       "      <td>T1053</td>\n",
       "      <td>005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>has the ability to launch scheduled tasks to e...</td>\n",
       "      <td>Scheduled Task</td>\n",
       "      <td>T1053</td>\n",
       "      <td>005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>has used scheduled tasks to maintain persistence.</td>\n",
       "      <td>Scheduled Task</td>\n",
       "      <td>T1053</td>\n",
       "      <td>005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4522</th>\n",
       "      <td>The men's S8 50 meters freestyle competition o...</td>\n",
       "      <td>none</td>\n",
       "      <td>none</td>\n",
       "      <td>none</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4523</th>\n",
       "      <td>Network data encryption Both trojans encrypt t...</td>\n",
       "      <td>none</td>\n",
       "      <td>none</td>\n",
       "      <td>none</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4524</th>\n",
       "      <td>has the ability to generate new C2 domains.</td>\n",
       "      <td>Fallback Channels</td>\n",
       "      <td>none</td>\n",
       "      <td>none</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4525</th>\n",
       "      <td>may create a file containing the results of th...</td>\n",
       "      <td>System Network Configuration Discovery</td>\n",
       "      <td>none</td>\n",
       "      <td>none</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4526</th>\n",
       "      <td>It is most known for its association football ...</td>\n",
       "      <td>none</td>\n",
       "      <td>none</td>\n",
       "      <td>none</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4527 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text   \n",
       "0     has overwritten the function pointer in the ex...  \\\n",
       "1     overwrites Explorers Shell_TrayWnd extra windo...   \n",
       "2     has used scheduled tasks to maintain persistence.   \n",
       "3     has the ability to launch scheduled tasks to e...   \n",
       "4     has used scheduled tasks to maintain persistence.   \n",
       "...                                                 ...   \n",
       "4522  The men's S8 50 meters freestyle competition o...   \n",
       "4523  Network data encryption Both trojans encrypt t...   \n",
       "4524        has the ability to generate new C2 domains.   \n",
       "4525  may create a file containing the results of th...   \n",
       "4526  It is most known for its association football ...   \n",
       "\n",
       "                              technique_name attack_id subclass_id  \n",
       "0              Extra Window Memory Injection     T1055         011  \n",
       "1              Extra Window Memory Injection     T1055         011  \n",
       "2                             Scheduled Task     T1053         005  \n",
       "3                             Scheduled Task     T1053         005  \n",
       "4                             Scheduled Task     T1053         005  \n",
       "...                                      ...       ...         ...  \n",
       "4522                                    none      none        none  \n",
       "4523                                    none      none        none  \n",
       "4524                       Fallback Channels      none        none  \n",
       "4525  System Network Configuration Discovery      none        none  \n",
       "4526                                    none      none        none  \n",
       "\n",
       "[4527 rows x 4 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classes_of_interest = ['T1041', 'T1106', 'T1082', 'T1033', 'T1112', 'T1070', 'T1090', 'T1021', 'T1218', 'T1095', 'T1548', 'T1053', 'T1071', 'T1574', 'T1562', 'T1204', 'T1012', 'T1140', 'T1055', 'T1105', 'T1552', 'T1486', 'T1083', 'T1078', 'T1047', 'T1190', 'T1543', 'T1113', 'T1003', 'T1059', 'T1057', 'T1027', 'T1219', 'T1036', 'T1005']\n",
    "positive_data = df[df['attack_id'].isin(classes_of_interest)]\n",
    "negative_data = df[df['attack_id'].isna()].sample(1000).fillna('none')\n",
    "data = pd.concat((positive_data, negative_data)).reset_index(drop=True)\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2ff7aba-e114-4ddc-934c-c197c0e5192d",
   "metadata": {},
   "source": [
    "The first model, fasttext, requires the training data to be inputted in an unusual way, namely by reading a text file where each line is a training instance, and the line starts with `__label__TAG`, where `TAG` is the tag associated with that line."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2a9aab65-2606-4f4c-9e6a-e3ffd4e5dc64",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       __label__T1055 has overwritten the function po...\n",
       "1       __label__T1055 overwrites Explorers Shell_Tray...\n",
       "2       __label__T1053 has used scheduled tasks to mai...\n",
       "3       __label__T1053 has the ability to launch sched...\n",
       "4       __label__T1053 has used scheduled tasks to mai...\n",
       "                              ...                        \n",
       "4522    __label__none The men's S8 50 meters freestyle...\n",
       "4523    __label__none Network data encryption Both tro...\n",
       "4524    __label__none has the ability to generate new ...\n",
       "4525    __label__none may create a file containing the...\n",
       "4526    __label__none It is most known for its associa...\n",
       "Length: 4527, dtype: object"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_text = '__label__' + data['attack_id'] + ' ' + data['text'].str.replace('\\n', ' ')\n",
    "data['fasttext_text'] = data_text\n",
    "data_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d3488721-61de-4bfd-ad18-7b7b5efdcda4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train_df, test_df = train_test_split(data, test_size=0.2, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "29773308-dd4c-49e9-8a96-7abe9842774c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train</th>\n",
       "      <th>test</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>attack_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>none</th>\n",
       "      <td>817</td>\n",
       "      <td>183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1059</th>\n",
       "      <td>541</td>\n",
       "      <td>165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1071</th>\n",
       "      <td>303</td>\n",
       "      <td>94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1070</th>\n",
       "      <td>297</td>\n",
       "      <td>71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1036</th>\n",
       "      <td>190</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1204</th>\n",
       "      <td>173</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1218</th>\n",
       "      <td>171</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1027</th>\n",
       "      <td>164</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1053</th>\n",
       "      <td>128</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1543</th>\n",
       "      <td>120</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1021</th>\n",
       "      <td>111</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1003</th>\n",
       "      <td>110</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1562</th>\n",
       "      <td>102</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1055</th>\n",
       "      <td>98</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1574</th>\n",
       "      <td>93</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1090</th>\n",
       "      <td>58</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1552</th>\n",
       "      <td>53</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1548</th>\n",
       "      <td>52</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1078</th>\n",
       "      <td>40</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           train  test\n",
       "attack_id             \n",
       "none         817   183\n",
       "T1059        541   165\n",
       "T1071        303    94\n",
       "T1070        297    71\n",
       "T1036        190    54\n",
       "T1204        173    48\n",
       "T1218        171    37\n",
       "T1027        164    36\n",
       "T1053        128    29\n",
       "T1543        120    33\n",
       "T1021        111    29\n",
       "T1003        110    26\n",
       "T1562        102    24\n",
       "T1055         98    22\n",
       "T1574         93    18\n",
       "T1090         58    14\n",
       "T1552         53     7\n",
       "T1548         52    12\n",
       "T1078         40     4"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_class_counts = train_df['attack_id'].value_counts(dropna=False)\n",
    "test_class_counts = test_df['attack_id'].value_counts(dropna=False)\n",
    "pd.concat({'train': train_class_counts, 'test': test_class_counts}, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d25f257b-675f-40a7-83a6-7a706f633155",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Read 0M words\n",
      "Number of words:  9723\n",
      "Number of labels: 19\n",
      "Progress: 100.0% words/sec/thread:  290784 lr:  0.000000 avg.loss:  0.086864 ETA:   0h 0m 0s\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "import fasttext\n",
    "\n",
    "train_data_file = Path('train_data.txt')\n",
    "train_data_file.write_text(train_df['fasttext_text'].str.cat(sep='\\n'))\n",
    "\n",
    "model = fasttext.train_supervised(input=str(train_data_file), epoch=200, lr=1.0, wordNgrams=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e9eab35-38ad-45a1-87f0-15a067815ce5",
   "metadata": {},
   "source": [
    "The way that the predictions are outputted is also unusual, so extra code is required to transform them into an ideal structure."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0fb91527-a138-4584-a972-b441cceb0442",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(('__label__none',), array([0.57492971]))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict(\"hello world\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bc41bb43-6847-4042-a131-bd095b3dfba5",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prediction</th>\n",
       "      <th>confidence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4066</th>\n",
       "      <td>T1070</td>\n",
       "      <td>0.616549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1751</th>\n",
       "      <td>T1562</td>\n",
       "      <td>0.877122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1580</th>\n",
       "      <td>T1059</td>\n",
       "      <td>1.000010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4168</th>\n",
       "      <td>none</td>\n",
       "      <td>0.999513</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1091</th>\n",
       "      <td>T1562</td>\n",
       "      <td>0.999987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2678</th>\n",
       "      <td>T1071</td>\n",
       "      <td>1.000010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2220</th>\n",
       "      <td>T1059</td>\n",
       "      <td>1.000010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2538</th>\n",
       "      <td>T1070</td>\n",
       "      <td>1.000010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4046</th>\n",
       "      <td>none</td>\n",
       "      <td>0.984438</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>T1053</td>\n",
       "      <td>0.944215</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>906 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     prediction  confidence\n",
       "4066      T1070    0.616549\n",
       "1751      T1562    0.877122\n",
       "1580      T1059    1.000010\n",
       "4168       none    0.999513\n",
       "1091      T1562    0.999987\n",
       "...         ...         ...\n",
       "2678      T1071    1.000010\n",
       "2220      T1059    1.000010\n",
       "2538      T1070    1.000010\n",
       "4046       none    0.984438\n",
       "49        T1053    0.944215\n",
       "\n",
       "[906 rows x 2 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds = test_df['text'].str.replace('\\n', ' ').apply(model.predict).apply(pd.Series).rename({0: 'prediction', 1: 'confidence'}, axis=1)\n",
    "preds['prediction'] = preds['prediction'].apply(lambda x: x[0].removeprefix('__label__'))\n",
    "preds['confidence'] = preds['confidence'].apply(lambda x: x[0])\n",
    "preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "fd61563a-7dd8-4f6c-b899-8c2f8211ee7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_recall_fscore_support as calculate_score\n",
    "\n",
    "def calculate_scores_df(actual: list[str], predicted: list[str]):\n",
    "    scores = calculate_score(actual, predicted)\n",
    "    scores_df = pd.DataFrame(scores).T\n",
    "    scores_df.columns = ['P', 'R', 'F1', '#']\n",
    "    scores_df.index = sorted(set(actual) | set(predicted))\n",
    "    scores_df.loc['(micro)'] = calculate_score(actual, predicted, average='micro')\n",
    "    scores_df.loc['(macro)'] = calculate_score(actual, predicted, average='macro')\n",
    "    return scores_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "65da5111-dc8c-4430-a3c2-b260d681b8ec",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>P</th>\n",
       "      <th>R</th>\n",
       "      <th>F1</th>\n",
       "      <th>#</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>T1003</th>\n",
       "      <td>0.961538</td>\n",
       "      <td>0.961538</td>\n",
       "      <td>0.961538</td>\n",
       "      <td>26.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1021</th>\n",
       "      <td>0.840000</td>\n",
       "      <td>0.724138</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>29.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1027</th>\n",
       "      <td>0.744186</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>0.810127</td>\n",
       "      <td>36.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1036</th>\n",
       "      <td>0.843137</td>\n",
       "      <td>0.796296</td>\n",
       "      <td>0.819048</td>\n",
       "      <td>54.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1053</th>\n",
       "      <td>0.888889</td>\n",
       "      <td>0.827586</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>29.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1055</th>\n",
       "      <td>0.894737</td>\n",
       "      <td>0.772727</td>\n",
       "      <td>0.829268</td>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1059</th>\n",
       "      <td>0.898810</td>\n",
       "      <td>0.915152</td>\n",
       "      <td>0.906907</td>\n",
       "      <td>165.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1070</th>\n",
       "      <td>0.784810</td>\n",
       "      <td>0.873239</td>\n",
       "      <td>0.826667</td>\n",
       "      <td>71.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1071</th>\n",
       "      <td>0.978022</td>\n",
       "      <td>0.946809</td>\n",
       "      <td>0.962162</td>\n",
       "      <td>94.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1078</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1090</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.923077</td>\n",
       "      <td>14.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1204</th>\n",
       "      <td>0.923077</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.960000</td>\n",
       "      <td>48.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1218</th>\n",
       "      <td>0.758621</td>\n",
       "      <td>0.594595</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>37.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1543</th>\n",
       "      <td>0.933333</td>\n",
       "      <td>0.848485</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>33.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1548</th>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1552</th>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.285714</td>\n",
       "      <td>0.363636</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1562</th>\n",
       "      <td>0.724138</td>\n",
       "      <td>0.875000</td>\n",
       "      <td>0.792453</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1574</th>\n",
       "      <td>0.866667</td>\n",
       "      <td>0.722222</td>\n",
       "      <td>0.787879</td>\n",
       "      <td>18.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>none</th>\n",
       "      <td>0.812500</td>\n",
       "      <td>0.852459</td>\n",
       "      <td>0.832000</td>\n",
       "      <td>183.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(micro)</th>\n",
       "      <td>0.858720</td>\n",
       "      <td>0.858720</td>\n",
       "      <td>0.858720</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(macro)</th>\n",
       "      <td>0.851884</td>\n",
       "      <td>0.793438</td>\n",
       "      <td>0.813960</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                P         R        F1      #\n",
       "T1003    0.961538  0.961538  0.961538   26.0\n",
       "T1021    0.840000  0.724138  0.777778   29.0\n",
       "T1027    0.744186  0.888889  0.810127   36.0\n",
       "T1036    0.843137  0.796296  0.819048   54.0\n",
       "T1053    0.888889  0.827586  0.857143   29.0\n",
       "T1055    0.894737  0.772727  0.829268   22.0\n",
       "T1059    0.898810  0.915152  0.906907  165.0\n",
       "T1070    0.784810  0.873239  0.826667   71.0\n",
       "T1071    0.978022  0.946809  0.962162   94.0\n",
       "T1078    1.000000  0.500000  0.666667    4.0\n",
       "T1090    1.000000  0.857143  0.923077   14.0\n",
       "T1204    0.923077  1.000000  0.960000   48.0\n",
       "T1218    0.758621  0.594595  0.666667   37.0\n",
       "T1543    0.933333  0.848485  0.888889   33.0\n",
       "T1548    0.833333  0.833333  0.833333   12.0\n",
       "T1552    0.500000  0.285714  0.363636    7.0\n",
       "T1562    0.724138  0.875000  0.792453   24.0\n",
       "T1574    0.866667  0.722222  0.787879   18.0\n",
       "none     0.812500  0.852459  0.832000  183.0\n",
       "(micro)  0.858720  0.858720  0.858720    NaN\n",
       "(macro)  0.851884  0.793438  0.813960    NaN"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fasttext_scores = calculate_scores_df(test_df['attack_id'].tolist(), preds['prediction'].tolist())\n",
    "fasttext_scores"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cfba716a-84fc-46dc-8975-9bb840ffc061",
   "metadata": {},
   "source": [
    "The technical report for the previous iteration of TRAM says this about the parameters used for logistic regression.\n",
    "\n",
    "> The logistic regression parameter settings include: using the Scikit Learning CountVectorizer as a feature generator, with a minimum document frequency for any given word of 3, removing “stopwords,” and retaining the case of words as they appeared in the input text (that is, not normalizing to a common case, such as lowercase)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0151fb79-2960-48ff-a04b-9e8f9e061fa9",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>CountVectorizer(lowercase=False, min_df=3, stop_words=&#x27;english&#x27;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">CountVectorizer</label><div class=\"sk-toggleable__content\"><pre>CountVectorizer(lowercase=False, min_df=3, stop_words=&#x27;english&#x27;)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "CountVectorizer(lowercase=False, min_df=3, stop_words='english')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "count_vectorizer = CountVectorizer(lowercase=False, stop_words='english', min_df=3)\n",
    "count_vectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b2421f7e-9424-4653-9bff-65eca009250a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['example', 'sentence'], dtype=object)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result = count_vectorizer.fit_transform([\"this is the example sentence.\", \"this is the next example sentence\", \"this is the last example sentence\"])\n",
    "count_vectorizer.get_feature_names_out()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d0301258-ed75-4e0b-aefb-a91062df6470",
   "metadata": {},
   "source": [
    "Note that \"example\" and \"sentence\" are the only words considered by the count vectorizer because \"this is the\" are all stop words, and \"next\" and \"last\" have a document frequency of only one each (less than three). We will re-fit the count vectorizer on the actual data in the next cell."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "cd88c97f-4ab8-4a38-bebe-d48fc7cb66e3",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>P</th>\n",
       "      <th>R</th>\n",
       "      <th>F1</th>\n",
       "      <th>#</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>T1003</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.846154</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>26.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1021</th>\n",
       "      <td>0.869565</td>\n",
       "      <td>0.689655</td>\n",
       "      <td>0.769231</td>\n",
       "      <td>29.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1027</th>\n",
       "      <td>0.794118</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.771429</td>\n",
       "      <td>36.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1036</th>\n",
       "      <td>0.808511</td>\n",
       "      <td>0.703704</td>\n",
       "      <td>0.752475</td>\n",
       "      <td>54.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1053</th>\n",
       "      <td>0.960000</td>\n",
       "      <td>0.827586</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>29.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1055</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.863636</td>\n",
       "      <td>0.926829</td>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1059</th>\n",
       "      <td>0.932515</td>\n",
       "      <td>0.921212</td>\n",
       "      <td>0.926829</td>\n",
       "      <td>165.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1070</th>\n",
       "      <td>0.871429</td>\n",
       "      <td>0.859155</td>\n",
       "      <td>0.865248</td>\n",
       "      <td>71.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1071</th>\n",
       "      <td>0.956989</td>\n",
       "      <td>0.946809</td>\n",
       "      <td>0.951872</td>\n",
       "      <td>94.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1078</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1090</th>\n",
       "      <td>0.916667</td>\n",
       "      <td>0.785714</td>\n",
       "      <td>0.846154</td>\n",
       "      <td>14.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1204</th>\n",
       "      <td>0.921569</td>\n",
       "      <td>0.979167</td>\n",
       "      <td>0.949495</td>\n",
       "      <td>48.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1218</th>\n",
       "      <td>0.852941</td>\n",
       "      <td>0.783784</td>\n",
       "      <td>0.816901</td>\n",
       "      <td>37.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1543</th>\n",
       "      <td>0.787879</td>\n",
       "      <td>0.787879</td>\n",
       "      <td>0.787879</td>\n",
       "      <td>33.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1548</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1552</th>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1562</th>\n",
       "      <td>0.772727</td>\n",
       "      <td>0.708333</td>\n",
       "      <td>0.739130</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1574</th>\n",
       "      <td>0.875000</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.823529</td>\n",
       "      <td>18.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>none</th>\n",
       "      <td>0.735426</td>\n",
       "      <td>0.896175</td>\n",
       "      <td>0.807882</td>\n",
       "      <td>183.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(micro)</th>\n",
       "      <td>0.855408</td>\n",
       "      <td>0.855408</td>\n",
       "      <td>0.855408</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(macro)</th>\n",
       "      <td>0.871333</td>\n",
       "      <td>0.786746</td>\n",
       "      <td>0.820925</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                P         R        F1      #\n",
       "T1003    1.000000  0.846154  0.916667   26.0\n",
       "T1021    0.869565  0.689655  0.769231   29.0\n",
       "T1027    0.794118  0.750000  0.771429   36.0\n",
       "T1036    0.808511  0.703704  0.752475   54.0\n",
       "T1053    0.960000  0.827586  0.888889   29.0\n",
       "T1055    1.000000  0.863636  0.926829   22.0\n",
       "T1059    0.932515  0.921212  0.926829  165.0\n",
       "T1070    0.871429  0.859155  0.865248   71.0\n",
       "T1071    0.956989  0.946809  0.951872   94.0\n",
       "T1078    1.000000  0.500000  0.666667    4.0\n",
       "T1090    0.916667  0.785714  0.846154   14.0\n",
       "T1204    0.921569  0.979167  0.949495   48.0\n",
       "T1218    0.852941  0.783784  0.816901   37.0\n",
       "T1543    0.787879  0.787879  0.787879   33.0\n",
       "T1548    1.000000  0.750000  0.857143   12.0\n",
       "T1552    0.500000  0.571429  0.533333    7.0\n",
       "T1562    0.772727  0.708333  0.739130   24.0\n",
       "T1574    0.875000  0.777778  0.823529   18.0\n",
       "none     0.735426  0.896175  0.807882  183.0\n",
       "(micro)  0.855408  0.855408  0.855408    NaN\n",
       "(macro)  0.871333  0.786746  0.820925    NaN"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "lr = LogisticRegression()\n",
    "lr.fit(count_vectorizer.fit_transform(train_df['text']), train_df['attack_id'])\n",
    "preds = lr.predict(count_vectorizer.transform(test_df['text']))\n",
    "lr_scores = calculate_scores_df(test_df['attack_id'], preds)\n",
    "lr_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8a1d7872-1bd1-4339-a292-7c3a1d4fa2a6",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"4\" halign=\"left\">fasttext</th>\n",
       "      <th colspan=\"4\" halign=\"left\">logistic_regression</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>P</th>\n",
       "      <th>R</th>\n",
       "      <th>F1</th>\n",
       "      <th>#</th>\n",
       "      <th>P</th>\n",
       "      <th>R</th>\n",
       "      <th>F1</th>\n",
       "      <th>#</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>T1003</th>\n",
       "      <td>0.961538</td>\n",
       "      <td>0.961538</td>\n",
       "      <td>0.961538</td>\n",
       "      <td>26.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.846154</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>26.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1021</th>\n",
       "      <td>0.840000</td>\n",
       "      <td>0.724138</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>29.0</td>\n",
       "      <td>0.869565</td>\n",
       "      <td>0.689655</td>\n",
       "      <td>0.769231</td>\n",
       "      <td>29.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1027</th>\n",
       "      <td>0.744186</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>0.810127</td>\n",
       "      <td>36.0</td>\n",
       "      <td>0.794118</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.771429</td>\n",
       "      <td>36.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1036</th>\n",
       "      <td>0.843137</td>\n",
       "      <td>0.796296</td>\n",
       "      <td>0.819048</td>\n",
       "      <td>54.0</td>\n",
       "      <td>0.808511</td>\n",
       "      <td>0.703704</td>\n",
       "      <td>0.752475</td>\n",
       "      <td>54.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1053</th>\n",
       "      <td>0.888889</td>\n",
       "      <td>0.827586</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>29.0</td>\n",
       "      <td>0.960000</td>\n",
       "      <td>0.827586</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>29.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1055</th>\n",
       "      <td>0.894737</td>\n",
       "      <td>0.772727</td>\n",
       "      <td>0.829268</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.863636</td>\n",
       "      <td>0.926829</td>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1059</th>\n",
       "      <td>0.898810</td>\n",
       "      <td>0.915152</td>\n",
       "      <td>0.906907</td>\n",
       "      <td>165.0</td>\n",
       "      <td>0.932515</td>\n",
       "      <td>0.921212</td>\n",
       "      <td>0.926829</td>\n",
       "      <td>165.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1070</th>\n",
       "      <td>0.784810</td>\n",
       "      <td>0.873239</td>\n",
       "      <td>0.826667</td>\n",
       "      <td>71.0</td>\n",
       "      <td>0.871429</td>\n",
       "      <td>0.859155</td>\n",
       "      <td>0.865248</td>\n",
       "      <td>71.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1071</th>\n",
       "      <td>0.978022</td>\n",
       "      <td>0.946809</td>\n",
       "      <td>0.962162</td>\n",
       "      <td>94.0</td>\n",
       "      <td>0.956989</td>\n",
       "      <td>0.946809</td>\n",
       "      <td>0.951872</td>\n",
       "      <td>94.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1078</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1090</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.923077</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>0.785714</td>\n",
       "      <td>0.846154</td>\n",
       "      <td>14.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1204</th>\n",
       "      <td>0.923077</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.960000</td>\n",
       "      <td>48.0</td>\n",
       "      <td>0.921569</td>\n",
       "      <td>0.979167</td>\n",
       "      <td>0.949495</td>\n",
       "      <td>48.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1218</th>\n",
       "      <td>0.758621</td>\n",
       "      <td>0.594595</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>37.0</td>\n",
       "      <td>0.852941</td>\n",
       "      <td>0.783784</td>\n",
       "      <td>0.816901</td>\n",
       "      <td>37.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1543</th>\n",
       "      <td>0.933333</td>\n",
       "      <td>0.848485</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>33.0</td>\n",
       "      <td>0.787879</td>\n",
       "      <td>0.787879</td>\n",
       "      <td>0.787879</td>\n",
       "      <td>33.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1548</th>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1552</th>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.285714</td>\n",
       "      <td>0.363636</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1562</th>\n",
       "      <td>0.724138</td>\n",
       "      <td>0.875000</td>\n",
       "      <td>0.792453</td>\n",
       "      <td>24.0</td>\n",
       "      <td>0.772727</td>\n",
       "      <td>0.708333</td>\n",
       "      <td>0.739130</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1574</th>\n",
       "      <td>0.866667</td>\n",
       "      <td>0.722222</td>\n",
       "      <td>0.787879</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.875000</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.823529</td>\n",
       "      <td>18.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>none</th>\n",
       "      <td>0.812500</td>\n",
       "      <td>0.852459</td>\n",
       "      <td>0.832000</td>\n",
       "      <td>183.0</td>\n",
       "      <td>0.735426</td>\n",
       "      <td>0.896175</td>\n",
       "      <td>0.807882</td>\n",
       "      <td>183.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(micro)</th>\n",
       "      <td>0.858720</td>\n",
       "      <td>0.858720</td>\n",
       "      <td>0.858720</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.855408</td>\n",
       "      <td>0.855408</td>\n",
       "      <td>0.855408</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(macro)</th>\n",
       "      <td>0.851884</td>\n",
       "      <td>0.793438</td>\n",
       "      <td>0.813960</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.871333</td>\n",
       "      <td>0.786746</td>\n",
       "      <td>0.820925</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         fasttext                            logistic_regression             \n",
       "                P         R        F1      #                   P         R   \n",
       "T1003    0.961538  0.961538  0.961538   26.0            1.000000  0.846154  \\\n",
       "T1021    0.840000  0.724138  0.777778   29.0            0.869565  0.689655   \n",
       "T1027    0.744186  0.888889  0.810127   36.0            0.794118  0.750000   \n",
       "T1036    0.843137  0.796296  0.819048   54.0            0.808511  0.703704   \n",
       "T1053    0.888889  0.827586  0.857143   29.0            0.960000  0.827586   \n",
       "T1055    0.894737  0.772727  0.829268   22.0            1.000000  0.863636   \n",
       "T1059    0.898810  0.915152  0.906907  165.0            0.932515  0.921212   \n",
       "T1070    0.784810  0.873239  0.826667   71.0            0.871429  0.859155   \n",
       "T1071    0.978022  0.946809  0.962162   94.0            0.956989  0.946809   \n",
       "T1078    1.000000  0.500000  0.666667    4.0            1.000000  0.500000   \n",
       "T1090    1.000000  0.857143  0.923077   14.0            0.916667  0.785714   \n",
       "T1204    0.923077  1.000000  0.960000   48.0            0.921569  0.979167   \n",
       "T1218    0.758621  0.594595  0.666667   37.0            0.852941  0.783784   \n",
       "T1543    0.933333  0.848485  0.888889   33.0            0.787879  0.787879   \n",
       "T1548    0.833333  0.833333  0.833333   12.0            1.000000  0.750000   \n",
       "T1552    0.500000  0.285714  0.363636    7.0            0.500000  0.571429   \n",
       "T1562    0.724138  0.875000  0.792453   24.0            0.772727  0.708333   \n",
       "T1574    0.866667  0.722222  0.787879   18.0            0.875000  0.777778   \n",
       "none     0.812500  0.852459  0.832000  183.0            0.735426  0.896175   \n",
       "(micro)  0.858720  0.858720  0.858720    NaN            0.855408  0.855408   \n",
       "(macro)  0.851884  0.793438  0.813960    NaN            0.871333  0.786746   \n",
       "\n",
       "                          \n",
       "               F1      #  \n",
       "T1003    0.916667   26.0  \n",
       "T1021    0.769231   29.0  \n",
       "T1027    0.771429   36.0  \n",
       "T1036    0.752475   54.0  \n",
       "T1053    0.888889   29.0  \n",
       "T1055    0.926829   22.0  \n",
       "T1059    0.926829  165.0  \n",
       "T1070    0.865248   71.0  \n",
       "T1071    0.951872   94.0  \n",
       "T1078    0.666667    4.0  \n",
       "T1090    0.846154   14.0  \n",
       "T1204    0.949495   48.0  \n",
       "T1218    0.816901   37.0  \n",
       "T1543    0.787879   33.0  \n",
       "T1548    0.857143   12.0  \n",
       "T1552    0.533333    7.0  \n",
       "T1562    0.739130   24.0  \n",
       "T1574    0.823529   18.0  \n",
       "none     0.807882  183.0  \n",
       "(micro)  0.855408    NaN  \n",
       "(macro)  0.820925    NaN  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "both = pd.concat({'fasttext': fasttext_scores, 'logistic_regression': lr_scores}, axis=1)\n",
    "both"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
